feat: белый список форматов + сводка батча по типам

This commit is contained in:
2026-06-28 10:21:33 +04:00
parent 684d1e7763
commit 992f833880
2 changed files with 19 additions and 2 deletions
+6 -2
View File
@@ -103,10 +103,11 @@ def classify_batch(batch_id, llm_client=None, repo=None):
garbage = 0
json_fixes = 0
json_total = 0
type_counts = {} # doc_type → count for batch summary
def _classify_one(doc):
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
nonlocal garbage, json_fixes, json_total
nonlocal garbage, json_fixes, json_total, type_counts
try:
# Stage 1: garbage by filename (0 tokens)
if _is_garbage_by_filename(doc["filename"]):
@@ -127,6 +128,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
json_total += 1
if needed_fix:
json_fixes += 1
dtype = result.get("doc_type", "other")
type_counts[dtype] = type_counts.get(dtype, 0) + 1
_db.set_classification(
doc["id"],
result.get("doc_type", "other"),
@@ -151,7 +154,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
failed += 1
return {"ok": True, "total": total, "done": done, "failed": failed,
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)}
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
"types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
def _safe_json_parse(raw):