lada_scraper: парсер phpBB форума нива-лада.рф + доку

lada_scraper/state.py

@@ -0,0 +1,48 @@
+"""Управление состоянием (state.json в папке раздела)."""
+
+import json, logging
+from pathlib import Path
+from .config import OUTPUT_DIR
+
+log = logging.getLogger(__name__)
+
+
+def _state_file(section_slug: str) -> Path:
+    d = OUTPUT_DIR / section_slug
+    d.mkdir(parents=True, exist_ok=True)
+    return d / "state.json"
+
+
+def load(section_slug: str) -> dict:
+    f = _state_file(section_slug)
+    if f.exists():
+        return json.loads(f.read_text(encoding="utf-8"))
+    return {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0}
+
+
+def save(section_slug: str, st: dict):
+    _state_file(section_slug).write_text(
+        json.dumps(st, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def merge_states(section_slug: str):
+    """Собрать состояния всех воркеров в единый state.json."""
+    root = OUTPUT_DIR / section_slug
+    merged = {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0}
+
+    for wdir in sorted(root.glob("worker_*/state.json")):
+        try:
+            wst = json.loads(wdir.read_text(encoding="utf-8"))
+            merged["topics_done"].update(wst.get("topics_done", {}))
+            merged["pages_done"].extend(wst.get("pages_done", []))
+            merged["topic_count"] += wst.get("topic_count", 0)
+        except Exception as e:
+            log.warning(f"⚠️  Ошибка чтения {wdir}: {e}")
+
+    merged["pages_done"] = sorted(set(merged["pages_done"]))
+    merged["file_index"] = len(list(root.glob("part_*.jsonl"))) - 1
+    if merged["file_index"] < 0:
+        merged["file_index"] = 0
+
+    _state_file(section_slug).write_text(
+        json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")