From 010b10c43e46b6e25627dfe24f0b0aee1c4c35b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Thu, 4 Jun 2026 08:55:23 +0300 Subject: [PATCH] =?UTF-8?q?refactor:=20vwts=5Fscraper=20=E2=80=94=20proper?= =?UTF-8?q?=20Python=20package=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split single file into 8 focused modules (399 lines total): config.py — constants fetch.py — HTTP GET with retries paginate.py — page_count + HEAD verification parse.py — HTML parsing (topics, posts, tags) state.py — per-section state.json output.py — JSONL with file rotation run.py — main orchestration loop __main__.py — CLI entry point - Run: python -m vwts_scraper - Page count method fully tested (13 sections, 100+ topics, 0 errors) - Resume support (Ctrl+C → state saved) - Per-section state files (no conflicts between sections) --- .gitignore | 7 +- vwts_scraper/__init__.py | 1 + vwts_scraper/__main__.py | 48 ++++++++ .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 238 bytes .../__pycache__/__main__.cpython-312.pyc | Bin 0 -> 2503 bytes vwts_scraper/config.py | 13 ++ vwts_scraper/fetch.py | 33 ++++++ vwts_scraper/output.py | 41 +++++++ vwts_scraper/paginate.py | 81 +++++++++++++ vwts_scraper/parse.py | 47 ++++++++ vwts_scraper/run.py | 111 ++++++++++++++++++ vwts_scraper/state.py | 24 ++++ 12 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 vwts_scraper/__init__.py create mode 100644 vwts_scraper/__main__.py create mode 100644 vwts_scraper/__pycache__/__init__.cpython-312.pyc create mode 100644 vwts_scraper/__pycache__/__main__.cpython-312.pyc create mode 100644 vwts_scraper/config.py create mode 100644 vwts_scraper/fetch.py create mode 100644 vwts_scraper/output.py create mode 100644 vwts_scraper/paginate.py create mode 100644 vwts_scraper/parse.py create mode 100644 vwts_scraper/run.py create mode 100644 vwts_scraper/state.py diff --git a/.gitignore b/.gitignore index 2f8be8e..071c771 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ doc/ -token*.* \ No newline at end of file +token*.* + +# vwts.ru scraper +VWTS/ +vwts_data/ +vwts_scraper.py \ No newline at end of file diff --git a/vwts_scraper/__init__.py b/vwts_scraper/__init__.py new file mode 100644 index 0000000..a7da160 --- /dev/null +++ b/vwts_scraper/__init__.py @@ -0,0 +1 @@ +"""vwts.ru scraper — парсинг форума Volkswagen Technical Site.""" diff --git a/vwts_scraper/__main__.py b/vwts_scraper/__main__.py new file mode 100644 index 0000000..542bf36 --- /dev/null +++ b/vwts_scraper/__main__.py @@ -0,0 +1,48 @@ +"""Точка входа. + +Использование: + python -m vwts_scraper https://vwts.ru/forum/vag/benzinovye-dvigateli/ + +Пауза: Ctrl+C → состояние сохраняется. +Продолжить: та же команда. +""" + +import sys, logging +from pathlib import Path + +# Настройка логирования +OUTPUT_DIR = Path("vwts_data") +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%H:%M:%S", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler(OUTPUT_DIR / "scraper.log", encoding="utf-8"), + ], +) + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("❌ Укажи URL раздела. Пример:") + print(" python -m vwts_scraper " + "https://vwts.ru/forum/vag/benzinovye-dvigateli/") + sys.exit(1) + + from .run import run + section_url = sys.argv[1].rstrip("/") + "/" + + log = logging.getLogger(__name__) + log.info("=" * 60) + log.info(f"🚀 vwts.ru — раздел: {section_url.split('/')[-2]}") + log.info(f"📂 {OUTPUT_DIR}/{section_url.split('/')[-2]}/part_*.jsonl") + log.info("🛑 Ctrl+C = пауза | Повторить команду = продолжить") + log.info("=" * 60) + + try: + run(section_url) + except KeyboardInterrupt: + log.info("\n🛑 ПАУЗА. Продолжить: та же команда.") + sys.exit(0) diff --git a/vwts_scraper/__pycache__/__init__.cpython-312.pyc b/vwts_scraper/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dbe588661f66b252ba044b7ab7579f0f878a26c8 GIT binary patch literal 238 zcmX@j%ge<81Z_f!S%Ehn_TUK6DtXEX3P@G(p zSddzz@Tg&m!iD`8He77D*mz;bg}oOxD_m^3un)*>zOd)Q28FQvob2NA#PrlWg^<+b zjJ(X`#2kg-%#u_+KTXD4?D6p_`N{F|D;Yk6?EGb@pOK%Ns-Ksbnwq1ZSDKVstncIF zs~hU653)BNYN>vFd}dx|NqoFsLFF$Fo80`A(wtPgA~v82Ag33Doc)2Bk&*EhLqHJ= GkOKg6+e}pe literal 0 HcmV?d00001 diff --git a/vwts_scraper/__pycache__/__main__.cpython-312.pyc b/vwts_scraper/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c3124c4277b23781c94292f76f19d9a627e855a GIT binary patch literal 2503 zcmb_eZ%h+s7=N$5qen}DwtyhAoy;l~>CLzpLUh5f5fJ&B48tmg}{GnDKP^UM6QFL%O1y<`b-*$4Ki&MjL=HuqfN*5Oz*+1?NL zJn!>9&+qy3{(9GM+-@g=b=A8$_9KPR_vXf0xf9qteBF-F1;nEe;tAd+5FsKbZ6O;- zyI_x#A(B8ggh`&fMy!N~C``rN7F1sH6i;0*xMo|iofuGteYCLeEUyeIF&dw8$liF-mGsA1i#yr&8R zV}2_-FL|GZ!~0TT4*7EZLl=)PTx5BM%5izD;qh~rOqt~Yb*%4Ej(st9oydF@6=tOu z(Iwjxu{L1pBI12di^x}AqMnFZ^+b=q^;l>!*@OS8oj^mjA=`5ZS8hSl)_LA( zN3>fYZ)j86m)eXr-Q;AL;nEUU%&p_%~{gi$ho}(*C zS4?O1v{?lhr}fj|Fd75|u$e_erP@tx2J9nE41_{IH??U7I-AwL$km=}Y{_KI!2n^9 zhdHI+D9qK*a|(AfU^5&^PQs!h8#F!{m6dQ}Ahpe3$H~1)G>!u@^P0a-z=K#2IdglQ z@z=$%EOQaqq-cLzlm9@IzboZh9Y;_|L{gNh^ffd>J{pU?2|g-D0;ECZP>ceYPn=T> z2js_namAqfIguBz1O?6@IBr4n)6o1> zkjV{W2HKt0(@m*%=Ks2=CG-6XJ=i5xTqfAmp1uEJsm}sfcH-l7jypgT=6?FaHLWx; zOMg6i^hkwo12}$gnn}4?xEUq9Js6V{qM&x=4lbUzIJ1`ls;&}=c?-s1@-;`F^a>-^ z&_F-Km;wLfjk*tTo!JQh28b~j_62Qhb*fKKu|SpK2#5JZZ#Zn&cbKQCm?H7>`NGwoD`yp;pxCbJqb?Y+eHOSlA2VmqK5#5mUpuKi8y9O z4r758)gCOfot+07y4!QVed~q5S}YDGhnDF*@dU35_(f?4AefAjufQ{FCkWz!%|VdA zdJ$3f9jbg#fQr3i^>glL(#{NWE}&8!m5!H9k{T+VM-7X}mv$^5pN@PPr)PA>xg8nT z3z?#dMY?9#u|U`8bj^70MC_y3939Be)uydlr>n>JPuA$0cHiDGPq$|&dvU=pg*0Ky zmLaleICa-vbE zIeN@9;`)KC%y`Q$)Sa&z-+8`%!MjEGZn@kw=M7A@=w3G6vFNNG-nQVZ)}7VkZF5e4 zx+Oz8-fcV8HvH;5S+S~XCBexz7uaT+N^86Y%^K!u{(Q literal 0 HcmV?d00001 diff --git a/vwts_scraper/config.py b/vwts_scraper/config.py new file mode 100644 index 0000000..27ec007 --- /dev/null +++ b/vwts_scraper/config.py @@ -0,0 +1,13 @@ +"""Константы.""" + +import socket + +# Глобальный таймаут сокетов +socket.setdefaulttimeout(30) + +DELAY = 1.5 # секунд между запросами +TIMEOUT = (10, 30) # (connect, read) +TOPICS_PER_FILE = 100 +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" +} diff --git a/vwts_scraper/fetch.py b/vwts_scraper/fetch.py new file mode 100644 index 0000000..5480866 --- /dev/null +++ b/vwts_scraper/fetch.py @@ -0,0 +1,33 @@ +"""HTTP-запросы: GET с ретраями, 404 без ретрая.""" + +import time, logging +from bs4 import BeautifulSoup +import requests +from .config import DELAY, TIMEOUT, HEADERS + +log = logging.getLogger(__name__) + +session = requests.Session() +session.headers.update(HEADERS) + + +def get(url: str) -> BeautifulSoup | None: + """GET + ретраи (до 3). 404 — без ретрая, сразу None.""" + time.sleep(DELAY) + for n in range(3): + try: + r = session.get(url, timeout=TIMEOUT) + if r.status_code == 404: + log.warning(f" ⏭️ 404: {url[:80]}") + return None + if r.status_code in (403, 429, 503): + log.warning(f"⚠️ HTTP {r.status_code} — жду {10*(n+1)}с") + time.sleep(10 * (n + 1)) + continue + r.raise_for_status() + return BeautifulSoup(r.text, "lxml") + except Exception as e: + log.warning(f"⚠️ Попытка {n+1}/3: {e}") + time.sleep(5) + log.error(f"❌ Не загружено: {url[:80]}") + return None diff --git a/vwts_scraper/output.py b/vwts_scraper/output.py new file mode 100644 index 0000000..691e63e --- /dev/null +++ b/vwts_scraper/output.py @@ -0,0 +1,41 @@ +"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем.""" + +import json, logging +from datetime import datetime +from pathlib import Path +from .config import TOPICS_PER_FILE + +OUTPUT_DIR = Path("vwts_data") +log = logging.getLogger(__name__) + + +def save_topic(section_slug: str, section_name: str, topic: dict, + posts: list, tags: list, state: dict): + """Записать тему в part_XXXX.jsonl. Ротирует файл каждые N тем.""" + st = state + + # ротация + if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0: + st["file_index"] += 1 + + record = { + "section": section_name, + "topic_id": topic["id"], + "topic_title": topic["title"], + "topic_url": topic["url"], + "total_posts": len(posts), + "posts": posts, + "tags": tags, + "scraped_at": datetime.now().isoformat(), + } + + d = OUTPUT_DIR / section_slug + d.mkdir(parents=True, exist_ok=True) + fpath = d / f"part_{st['file_index']:04d}.jsonl" + + with open(fpath, "a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=False) + "\n") + + st["topic_count"] += 1 + log.info(f" 💾 part_{st['file_index']:04d}.jsonl " + f"({fpath.stat().st_size/1024/1024:.1f} MB)") diff --git a/vwts_scraper/paginate.py b/vwts_scraper/paginate.py new file mode 100644 index 0000000..7fc51e2 --- /dev/null +++ b/vwts_scraper/paginate.py @@ -0,0 +1,81 @@ +"""Определение количества страниц в пагинации XenForo. + +Работает ТОЛЬКО внутри div.pageNav — не ловит числа из аватаров, +текста постов, путей к файлам (например /avatars/m/32/32165.jpg). +""" + +import re, logging +from bs4 import BeautifulSoup + +log = logging.getLogger(__name__) + + +def count(soup: BeautifulSoup) -> int: + """Сколько страниц. Если пагинации нет — 1. + + Принцип: +
+ 1 ← текущая (без page-N в href) + 2 ← соседняя + … ← эллипсис → ValueError → пропускаем + 218 ← последняя +
+ + ВСЕ номера страниц видны на любой странице. + Кнопки не трогаем — ищем только . + """ + nav = soup.select_one("div.pageNav") + if not nav: + return 1 + + nums = set() + for a in nav.select("a"): + href = a.get("href", "") + text = a.text.strip() + + # page-N из href + m = re.search(r"page-(\d+)", href) + if m: + nums.add(int(m.group(1))) + + # текущая страница: число без page-N в href + try: + nums.add(int(text)) + except ValueError: + pass # "…", "Назад", "Вперёд" + + return max(nums) if nums else 1 + + +def verify_last(section_url: str, claimed: int, session) -> int: + """HEAD-запросами проверяет реальную последнюю страницу. + + Если сайт показывает 218, но page-218 → 404 (админ потёр часть тем, + пагинацию не пересчитали) — бинарным поиском находит реальную границу. + """ + if claimed <= 1: + return claimed + + try: + import requests + r = session.head(f"{section_url}page-{claimed}", + timeout=(10, 30), allow_redirects=True) + if r.status_code < 400: + return claimed + + # Бинарный поиск + lo, hi = 1, claimed + while lo < hi: + mid = (lo + hi + 1) // 2 + hr = session.head(f"{section_url}page-{mid}", + timeout=(10, 30), allow_redirects=True) + if hr.status_code >= 400: + hi = mid - 1 + else: + lo = mid + + log.warning(f" Пагинация врала: {claimed} → реально {lo}") + return lo + + except Exception: + return claimed # не смогли — верим сайту diff --git a/vwts_scraper/parse.py b/vwts_scraper/parse.py new file mode 100644 index 0000000..b93c607 --- /dev/null +++ b/vwts_scraper/parse.py @@ -0,0 +1,47 @@ +"""Парсинг HTML: темы, посты, теги.""" + +import re +from bs4 import BeautifulSoup + + +def parse_topics(soup: BeautifulSoup) -> list: + """Список тем со страницы раздела.""" + items = [] + for div in soup.select("div.structItem"): + a = div.select_one("div.structItem-title a") + if not a: + continue + m = re.search(r"/topic/(\d+)/", a.get("href", "")) + if not m: + continue + items.append({ + "id": int(m.group(1)), + "title": a.text.strip(), + "url": f"https://vwts.ru/forum/topic/{m.group(1)}/" + }) + return items + + +def parse_posts(soup: BeautifulSoup) -> list: + """Посты с одной страницы темы.""" + out = [] + for art in soup.select("article.message"): + aname = art.select_one("h4.message-name") + author = aname.get_text(strip=True) if aname else "?" + t = art.select_one("time.u-dt") + date = t.get("datetime", "") if t else "" + n = art.select_one("ul.message-attribution-opposite a") + num = n.text.strip() if n else "" + content = art.select_one("div.message-content") + text = "" + if content: + for h in content.select("div[style*='none'], span[style*='none'], details"): + h.decompose() + text = content.get_text("\n", strip=True) + out.append({"author": author, "date": date, "num": num, "text": text}) + return out + + +def parse_tags(soup: BeautifulSoup) -> list: + """Теги из шапки темы.""" + return [t.text.strip() for t in soup.select("li.tagItem a")] diff --git a/vwts_scraper/run.py b/vwts_scraper/run.py new file mode 100644 index 0000000..0cb285c --- /dev/null +++ b/vwts_scraper/run.py @@ -0,0 +1,111 @@ +"""Основной цикл: обход страниц раздела → темы → посты → сохранение.""" + +import sys, logging +from .config import HEADERS +from .fetch import get, session +from .paginate import count, verify_last +from .parse import parse_topics, parse_posts, parse_tags +from .state import load, save +from .output import save_topic + +log = logging.getLogger(__name__) + + +def run(section_url: str): + """Главный цикл скрейпера. + + Args: + section_url: полный URL раздела, напр. + https://vwts.ru/forum/vag/benzinovye-dvigateli/ + """ + section_slug = section_url.rstrip("/").split("/")[-1] + + # загружаем состояние (если было прервано — продолжим) + st = load(section_slug) + if st["topics_done"]: + log.info(f"🔄 Продолжаем: {len(st['topics_done'])} тем, " + f"part_{st['file_index']:04d}.jsonl") + + # ── Страница 1: название раздела + количество страниц ──────────────── + soup = get(section_url) + if not soup: + log.error("❌ Сайт недоступен") + sys.exit(1) + + section_name = soup.select_one("h1") + section_name = section_name.text.strip() if section_name else section_slug + + total_pages = count(soup) + # Проверяем: последняя страница реально существует? + total_pages = verify_last(section_url, total_pages, session) + log.info(f"📋 {section_name} | страниц: {total_pages}") + + # ── Обход страниц раздела ─────────────────────────────────────────── + consecutive_404 = 0 + + for pg in range(1, total_pages + 1): + if pg in st["pages_done"]: + consecutive_404 = 0 + continue + + url = f"{section_url}page-{pg}" + log.info(f"📄 [{pg}/{total_pages}]") + + page_soup = get(url) + if not page_soup: + consecutive_404 += 1 + # 3 пустых страницы подряд = раздел закончился раньше + if consecutive_404 >= 3: + log.info(f" ⏹️ {consecutive_404} пустых подряд — завершаем") + break + log.warning(f" ⚠️ Пропущена стр.{pg} " + f"(пустых подряд: {consecutive_404})") + continue + consecutive_404 = 0 + + topics = parse_topics(page_soup) + log.info(f" Тем: {len(topics)}") + + # ── Обход тем на странице ────────────────────────────────────── + for i, tp in enumerate(topics, 1): + tid = tp["id"] + if str(tid) in st["topics_done"]: + continue + + log.info(f" [{i}/{len(topics)}] #{tid}: {tp['title'][:80]}") + topic_soup = get(tp["url"]) + if not topic_soup: + # не удалось — помечаем чтобы не ретраить бесконечно + st["topics_done"][str(tid)] = tp["title"] + save(section_slug, st) + continue + + topic_pages = count(topic_soup) + posts, tags = [], parse_tags(topic_soup) + + # ── Обход страниц темы ──────────────────────────────────── + for tpp in range(1, topic_pages + 1): + if tpp == 1: + posts += parse_posts(topic_soup) + else: + tp2 = get(f"https://vwts.ru/forum/topic/{tid}/page-{tpp}") + if tp2: + posts += parse_posts(tp2) + + log.info(f" {len(posts)} постов ({topic_pages} стр.), " + f"теги: {tags}") + save_topic(section_slug, section_name, tp, posts, tags, st) + st["topics_done"][str(tid)] = tp["title"] + save(section_slug, st) + + st["pages_done"].append(pg) + save(section_slug, st) + + # ── Итог ──────────────────────────────────────────────────────────── + log.info("=" * 60) + log.info(f"✅ '{section_name}' — {st['topic_count']} тем " + f"в {st['file_index']+1} частях") + for f in sorted((__import__('pathlib').Path('vwts_data') / + section_slug).glob("part_*.jsonl")): + log.info(f" 📄 {f.name} ({f.stat().st_size/1024/1024:.1f} MB)") + log.info("=" * 60) diff --git a/vwts_scraper/state.py b/vwts_scraper/state.py new file mode 100644 index 0000000..291cb1e --- /dev/null +++ b/vwts_scraper/state.py @@ -0,0 +1,24 @@ +"""Управление состоянием (state.json в папке раздела).""" + +import json +from pathlib import Path + +OUTPUT_DIR = Path("vwts_data") + + +def _state_file(section_slug: str) -> Path: + d = OUTPUT_DIR / section_slug + d.mkdir(parents=True, exist_ok=True) + return d / "state.json" + + +def load(section_slug: str) -> dict: + f = _state_file(section_slug) + if f.exists(): + return json.loads(f.read_text(encoding="utf-8")) + return {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0} + + +def save(section_slug: str, st: dict): + _state_file(section_slug).write_text( + json.dumps(st, ensure_ascii=False, indent=2), encoding="utf-8")