naeel
1cfa6d1595
- paginate.py: max(nums) on empty set → guard - paginate.py: HEAD fallback to GET if server doesn't support HEAD - output.py: disk health check every 500 topics - run.py: avoid page-1 extra redirect - __main__.py: URL validation (must contain vwts.ru)
50 lines
1.6 KiB
Python
50 lines
1.6 KiB
Python
"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем."""
|
|
|
|
import json, logging
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from .config import TOPICS_PER_FILE
|
|
|
|
OUTPUT_DIR = Path("vwts_data")
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def save_topic(section_slug: str, section_name: str, topic: dict,
|
|
posts: list, tags: list, state: dict):
|
|
"""Записать тему в part_XXXX.jsonl. Ротирует файл каждые N тем."""
|
|
st = state
|
|
|
|
# ротация
|
|
if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0:
|
|
st["file_index"] += 1
|
|
|
|
record = {
|
|
"section": section_name,
|
|
"topic_id": topic["id"],
|
|
"topic_title": topic["title"],
|
|
"topic_url": topic["url"],
|
|
"total_posts": len(posts),
|
|
"posts": posts,
|
|
"tags": tags,
|
|
"scraped_at": datetime.now().isoformat(),
|
|
}
|
|
|
|
d = OUTPUT_DIR / section_slug
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
fpath = d / f"part_{st['file_index']:04d}.jsonl"
|
|
|
|
with open(fpath, "a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
|
|
st["topic_count"] += 1
|
|
log.info(f" 💾 part_{st['file_index']:04d}.jsonl "
|
|
f"({fpath.stat().st_size/1024/1024:.1f} MB)")
|
|
|
|
if st["topic_count"] % 500 == 0:
|
|
# Каждые 500 тем — проверяем что диск ещё жив
|
|
try:
|
|
fpath.stat()
|
|
except OSError as e:
|
|
log.critical(f"❌ Ошибка диска: {e}")
|
|
raise
|