49 lines
1.7 KiB
Python
49 lines
1.7 KiB
Python
"""Управление состоянием (state.json в папке раздела)."""
|
|
|
|
import json, logging
|
|
from pathlib import Path
|
|
from .config import OUTPUT_DIR
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def _state_file(section_slug: str) -> Path:
|
|
d = OUTPUT_DIR / section_slug
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return d / "state.json"
|
|
|
|
|
|
def load(section_slug: str) -> dict:
|
|
f = _state_file(section_slug)
|
|
if f.exists():
|
|
return json.loads(f.read_text(encoding="utf-8"))
|
|
return {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0}
|
|
|
|
|
|
def save(section_slug: str, st: dict):
|
|
_state_file(section_slug).write_text(
|
|
json.dumps(st, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
|
def merge_states(section_slug: str):
|
|
"""Собрать состояния всех воркеров в единый state.json."""
|
|
root = OUTPUT_DIR / section_slug
|
|
merged = {"topics_done": {}, "pages_done": [], "file_index": 0, "topic_count": 0}
|
|
|
|
for wdir in sorted(root.glob("worker_*/state.json")):
|
|
try:
|
|
wst = json.loads(wdir.read_text(encoding="utf-8"))
|
|
merged["topics_done"].update(wst.get("topics_done", {}))
|
|
merged["pages_done"].extend(wst.get("pages_done", []))
|
|
merged["topic_count"] += wst.get("topic_count", 0)
|
|
except Exception as e:
|
|
log.warning(f"⚠️ Ошибка чтения {wdir}: {e}")
|
|
|
|
merged["pages_done"] = sorted(set(merged["pages_done"]))
|
|
merged["file_index"] = len(list(root.glob("part_*.jsonl"))) - 1
|
|
if merged["file_index"] < 0:
|
|
merged["file_index"] = 0
|
|
|
|
_state_file(section_slug).write_text(
|
|
json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|