lada_scraper: парсер phpBB форума нива-лада.рф + доку

lada_scraper/output.py

@@ -0,0 +1,44 @@
+"""Сохранение темы в JSONL. Ротация каждые TOPICS_PER_FILE тем."""
+
+import json, logging
+from datetime import datetime
+from pathlib import Path
+from .config import TOPICS_PER_FILE, OUTPUT_DIR
+
+log = logging.getLogger(__name__)
+
+
+def save_topic(section_slug: str, section_name: str, topic: dict,
+               posts: list, state: dict, worker_dir: Path = None):
+    """Записать тему в part_XXXX.jsonl."""
+    st = state
+
+    if st["topic_count"] > 0 and st["topic_count"] % TOPICS_PER_FILE == 0:
+        st["file_index"] += 1
+
+    record = {
+        "section": section_name,
+        "section_id": topic.get("forum_id"),
+        "topic_id": topic["id"],
+        "topic_title": topic["title"],
+        "topic_url": topic["url"],
+        "total_posts": len(posts),
+        "posts": posts,
+        "scraped_at": datetime.now().isoformat(),
+    }
+
+    d = worker_dir if worker_dir else (OUTPUT_DIR / section_slug)
+    d.mkdir(parents=True, exist_ok=True)
+    fpath = d / f"part_{st['file_index']:04d}.jsonl"
+
+    with open(fpath, "a", encoding="utf-8") as fh:
+        fh.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+    st["topic_count"] += 1
+
+    if st["topic_count"] % 500 == 0:
+        try:
+            fpath.stat()
+        except OSError as e:
+            log.critical(f"❌ Ошибка диска: {e}")
+            raise