diff --git a/vwts_scraper/__main__.py b/vwts_scraper/__main__.py index 3771ff0..b1009dd 100644 --- a/vwts_scraper/__main__.py +++ b/vwts_scraper/__main__.py @@ -8,10 +8,8 @@ """ import sys, logging -from pathlib import Path +from .config import OUTPUT_DIR -# Настройка логирования -OUTPUT_DIR = Path("vwts_data") OUTPUT_DIR.mkdir(parents=True, exist_ok=True) logging.basicConfig( diff --git a/vwts_scraper/__pycache__/__init__.cpython-312.pyc b/vwts_scraper/__pycache__/__init__.cpython-312.pyc index dbe5886..89813b7 100644 Binary files a/vwts_scraper/__pycache__/__init__.cpython-312.pyc and b/vwts_scraper/__pycache__/__init__.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/__main__.cpython-312.pyc b/vwts_scraper/__pycache__/__main__.cpython-312.pyc index 8c3124c..c60ae33 100644 Binary files a/vwts_scraper/__pycache__/__main__.cpython-312.pyc and b/vwts_scraper/__pycache__/__main__.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/config.cpython-312.pyc b/vwts_scraper/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..6af11cc Binary files /dev/null and b/vwts_scraper/__pycache__/config.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/fetch.cpython-312.pyc b/vwts_scraper/__pycache__/fetch.cpython-312.pyc new file mode 100644 index 0000000..f2f6b0f Binary files /dev/null and b/vwts_scraper/__pycache__/fetch.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/output.cpython-312.pyc b/vwts_scraper/__pycache__/output.cpython-312.pyc new file mode 100644 index 0000000..8961892 Binary files /dev/null and b/vwts_scraper/__pycache__/output.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/paginate.cpython-312.pyc b/vwts_scraper/__pycache__/paginate.cpython-312.pyc new file mode 100644 index 0000000..8f39559 Binary files /dev/null and b/vwts_scraper/__pycache__/paginate.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/parse.cpython-312.pyc b/vwts_scraper/__pycache__/parse.cpython-312.pyc new file mode 100644 index 0000000..712c028 Binary files /dev/null and b/vwts_scraper/__pycache__/parse.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/run.cpython-312.pyc b/vwts_scraper/__pycache__/run.cpython-312.pyc new file mode 100644 index 0000000..7043418 Binary files /dev/null and b/vwts_scraper/__pycache__/run.cpython-312.pyc differ diff --git a/vwts_scraper/__pycache__/state.cpython-312.pyc b/vwts_scraper/__pycache__/state.cpython-312.pyc new file mode 100644 index 0000000..6a18192 Binary files /dev/null and b/vwts_scraper/__pycache__/state.cpython-312.pyc differ diff --git a/vwts_scraper/config.py b/vwts_scraper/config.py index 27ec007..5df7f32 100644 --- a/vwts_scraper/config.py +++ b/vwts_scraper/config.py @@ -1,10 +1,12 @@ """Константы.""" import socket +from pathlib import Path # Глобальный таймаут сокетов socket.setdefaulttimeout(30) +OUTPUT_DIR = Path("vwts_data") # куда сохраняем данные DELAY = 1.5 # секунд между запросами TIMEOUT = (10, 30) # (connect, read) TOPICS_PER_FILE = 100 diff --git a/vwts_scraper/output.py b/vwts_scraper/output.py index d94ef7c..64e128b 100644 --- a/vwts_scraper/output.py +++ b/vwts_scraper/output.py @@ -2,10 +2,8 @@ import json, logging from datetime import datetime -from pathlib import Path -from .config import TOPICS_PER_FILE +from .config import TOPICS_PER_FILE, OUTPUT_DIR -OUTPUT_DIR = Path("vwts_data") log = logging.getLogger(__name__) diff --git a/vwts_scraper/run.py b/vwts_scraper/run.py index 208c040..b34e0b6 100644 --- a/vwts_scraper/run.py +++ b/vwts_scraper/run.py @@ -1,7 +1,9 @@ """Основной цикл: обход страниц раздела → темы → посты → сохранение.""" import sys, logging -from .config import HEADERS +from pathlib import Path +from urllib.parse import urlparse +from .config import OUTPUT_DIR from .fetch import get, session from .paginate import count, verify_last from .parse import parse_topics, parse_posts, parse_tags @@ -85,11 +87,13 @@ def run(section_url: str): posts, tags = [], parse_tags(topic_soup) # ── Обход страниц темы ──────────────────────────────────── + base = f"{urlparse(section_url).scheme}://{urlparse(section_url).netloc}" + for tpp in range(1, topic_pages + 1): if tpp == 1: posts += parse_posts(topic_soup) else: - tp2 = get(f"https://vwts.ru/forum/topic/{tid}/page-{tpp}") + tp2 = get(f"{base}/forum/topic/{tid}/page-{tpp}") if tp2: posts += parse_posts(tp2) @@ -106,7 +110,6 @@ def run(section_url: str): log.info("=" * 60) log.info(f"✅ '{section_name}' — {st['topic_count']} тем " f"в {st['file_index']+1} частях") - for f in sorted((__import__('pathlib').Path('vwts_data') / - section_slug).glob("part_*.jsonl")): + for f in sorted((OUTPUT_DIR / section_slug).glob("part_*.jsonl")): log.info(f" 📄 {f.name} ({f.stat().st_size/1024/1024:.1f} MB)") log.info("=" * 60) diff --git a/vwts_scraper/state.py b/vwts_scraper/state.py index 291cb1e..6917c95 100644 --- a/vwts_scraper/state.py +++ b/vwts_scraper/state.py @@ -1,9 +1,7 @@ """Управление состоянием (state.json в папке раздела).""" import json -from pathlib import Path - -OUTPUT_DIR = Path("vwts_data") +from .config import OUTPUT_DIR def _state_file(section_slug: str) -> Path: