"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем.""" import json, logging from datetime import datetime from pathlib import Path from .config import TOPICS_PER_FILE, OUTPUT_DIR log = logging.getLogger(__name__) def save_topic(section_slug: str, section_name: str, topic: dict, posts: list, state: dict, worker_dir: Path = None): """Записать тему в part_XXXX.jsonl.""" st = state if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0: st["file_index"] += 1 record = { "section": section_name, "section_id": topic.get("forum_id"), "topic_id": topic["id"], "topic_title": topic["title"], "topic_url": topic["url"], "total_posts": len(posts), "posts": posts, "scraped_at": datetime.now().isoformat(), } d = worker_dir if worker_dir else (OUTPUT_DIR / section_slug) d.mkdir(parents=True, exist_ok=True) fpath = d / f"part_{st['file_index']:04d}.jsonl" with open(fpath, "a", encoding="utf-8") as fh: fh.write(json.dumps(record, ensure_ascii=False) + "\n") st["topic_count"] += 1 if st["topic_count"] % 500 == 0: try: fpath.stat() except OSError as e: log.critical(f"❌ Ошибка диска: {e}") raise