"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем.""" import json, logging from datetime import datetime from pathlib import Path from .config import TOPICS_PER_FILE OUTPUT_DIR = Path("vwts_data") log = logging.getLogger(__name__) def save_topic(section_slug: str, section_name: str, topic: dict, posts: list, tags: list, state: dict): """Записать тему в part_XXXX.jsonl. Ротирует файл каждые N тем.""" st = state # ротация if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0: st["file_index"] += 1 record = { "section": section_name, "topic_id": topic["id"], "topic_title": topic["title"], "topic_url": topic["url"], "total_posts": len(posts), "posts": posts, "tags": tags, "scraped_at": datetime.now().isoformat(), } d = OUTPUT_DIR / section_slug d.mkdir(parents=True, exist_ok=True) fpath = d / f"part_{st['file_index']:04d}.jsonl" with open(fpath, "a", encoding="utf-8") as fh: fh.write(json.dumps(record, ensure_ascii=False) + "\n") st["topic_count"] += 1 log.info(f" 💾 part_{st['file_index']:04d}.jsonl " f"({fpath.stat().st_size/1024/1024:.1f} MB)")