feat: белый список форматов + сводка батча по типам
This commit is contained in:
@@ -103,10 +103,11 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
garbage = 0
|
||||
json_fixes = 0
|
||||
json_total = 0
|
||||
type_counts = {} # doc_type → count for batch summary
|
||||
|
||||
def _classify_one(doc):
|
||||
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
|
||||
nonlocal garbage, json_fixes, json_total
|
||||
nonlocal garbage, json_fixes, json_total, type_counts
|
||||
try:
|
||||
# Stage 1: garbage by filename (0 tokens)
|
||||
if _is_garbage_by_filename(doc["filename"]):
|
||||
@@ -127,6 +128,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
json_total += 1
|
||||
if needed_fix:
|
||||
json_fixes += 1
|
||||
dtype = result.get("doc_type", "other")
|
||||
type_counts[dtype] = type_counts.get(dtype, 0) + 1
|
||||
_db.set_classification(
|
||||
doc["id"],
|
||||
result.get("doc_type", "other"),
|
||||
@@ -151,7 +154,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
failed += 1
|
||||
|
||||
return {"ok": True, "total": total, "done": done, "failed": failed,
|
||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)}
|
||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
|
||||
"types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
|
||||
|
||||
|
||||
def _safe_json_parse(raw):
|
||||
|
||||
@@ -4,6 +4,15 @@ from db import documents, contracts, supplements
|
||||
from .parse import parse_file
|
||||
|
||||
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
|
||||
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "zip"}
|
||||
|
||||
|
||||
def _check_format(filename: str) -> str | None:
|
||||
"""Validate file extension. Returns error string or None."""
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))})"
|
||||
return None
|
||||
|
||||
|
||||
# ── Чистые функции (тестируемы без HTTP) ──────────────────────
|
||||
@@ -77,6 +86,10 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "",
|
||||
if not filename or not file_data:
|
||||
return {"ok": False, "error": "no file"}
|
||||
|
||||
fmt_err = _check_format(filename)
|
||||
if fmt_err:
|
||||
return {"ok": False, "error": fmt_err}
|
||||
|
||||
import hashlib
|
||||
content_hash = hashlib.sha256(file_data).hexdigest()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user