feat: белый список форматов + сводка батча по типам
This commit is contained in:
@@ -103,10 +103,11 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
garbage = 0
|
||||
json_fixes = 0
|
||||
json_total = 0
|
||||
type_counts = {} # doc_type → count for batch summary
|
||||
|
||||
def _classify_one(doc):
|
||||
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
|
||||
nonlocal garbage, json_fixes, json_total
|
||||
nonlocal garbage, json_fixes, json_total, type_counts
|
||||
try:
|
||||
# Stage 1: garbage by filename (0 tokens)
|
||||
if _is_garbage_by_filename(doc["filename"]):
|
||||
@@ -127,6 +128,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
json_total += 1
|
||||
if needed_fix:
|
||||
json_fixes += 1
|
||||
dtype = result.get("doc_type", "other")
|
||||
type_counts[dtype] = type_counts.get(dtype, 0) + 1
|
||||
_db.set_classification(
|
||||
doc["id"],
|
||||
result.get("doc_type", "other"),
|
||||
@@ -151,7 +154,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
failed += 1
|
||||
|
||||
return {"ok": True, "total": total, "done": done, "failed": failed,
|
||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)}
|
||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
|
||||
"types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
|
||||
|
||||
|
||||
def _safe_json_parse(raw):
|
||||
|
||||
Reference in New Issue
Block a user