feat: белый список форматов + сводка батча по типам

This commit is contained in:
2026-06-28 10:21:33 +04:00
parent 684d1e7763
commit 992f833880
2 changed files with 19 additions and 2 deletions
+6 -2
View File
@@ -103,10 +103,11 @@ def classify_batch(batch_id, llm_client=None, repo=None):
garbage = 0 garbage = 0
json_fixes = 0 json_fixes = 0
json_total = 0 json_total = 0
type_counts = {} # doc_type → count for batch summary
def _classify_one(doc): def _classify_one(doc):
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить.""" """Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
nonlocal garbage, json_fixes, json_total nonlocal garbage, json_fixes, json_total, type_counts
try: try:
# Stage 1: garbage by filename (0 tokens) # Stage 1: garbage by filename (0 tokens)
if _is_garbage_by_filename(doc["filename"]): if _is_garbage_by_filename(doc["filename"]):
@@ -127,6 +128,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
json_total += 1 json_total += 1
if needed_fix: if needed_fix:
json_fixes += 1 json_fixes += 1
dtype = result.get("doc_type", "other")
type_counts[dtype] = type_counts.get(dtype, 0) + 1
_db.set_classification( _db.set_classification(
doc["id"], doc["id"],
result.get("doc_type", "other"), result.get("doc_type", "other"),
@@ -151,7 +154,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
failed += 1 failed += 1
return {"ok": True, "total": total, "done": done, "failed": failed, return {"ok": True, "total": total, "done": done, "failed": failed,
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)} "garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
"types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
def _safe_json_parse(raw): def _safe_json_parse(raw):
+13
View File
@@ -4,6 +4,15 @@ from db import documents, contracts, supplements
from .parse import parse_file from .parse import parse_file
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "zip"}
def _check_format(filename: str) -> str | None:
"""Validate file extension. Returns error string or None."""
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
if ext not in ALLOWED_EXTENSIONS:
return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))})"
return None
# ── Чистые функции (тестируемы без HTTP) ────────────────────── # ── Чистые функции (тестируемы без HTTP) ──────────────────────
@@ -77,6 +86,10 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "",
if not filename or not file_data: if not filename or not file_data:
return {"ok": False, "error": "no file"} return {"ok": False, "error": "no file"}
fmt_err = _check_format(filename)
if fmt_err:
return {"ok": False, "error": fmt_err}
import hashlib import hashlib
content_hash = hashlib.sha256(file_data).hexdigest() content_hash = hashlib.sha256(file_data).hexdigest()